#!/bin/bash

MAIL="yajin@corp.ebay.com"
SERVERS="
        ddkitdns01.vip.its.ebay.com
        ddkitdns02.vip.its.ebay.com
        kulitdns01.vip.its.ebay.com
        kulitdns02.vip.its.ebay.com
        shaitdns01.vip.its.ebay.com
        shaitdns02.vip.its.ebay.com
        phxsapdns01.vip.get.ebay.com
        phxsapdns02.vip.get.ebay.com
"

LOG="/ebay/bind-logs/bind-misc.log"
ERROR="refresh: retry limit for master"

GREP="/usr/xpg4/bin/grep"

export PATH=/opt/bind/bin:/opt/bind/sbin:/bin:/usr/bin

hang=""

for s in $(echo $SERVERS); do

	typeset -i no
	ssh $s "tail -100 $LOG" | $GREP -q  "$ERROR"
	if [ $? -eq 0 ]; then 
		no=$(ssh $s "rndc status" | grep "soa queries in progress:" | awk -F":" '{print $2}')
		if [ $no -ge 50 ]; then hang="${hang} ${s}"; fi
	fi
done

if [ "X${hang}" != "X" ]; then
	if [ "X${SSH_TTY}" != "X" ]; then
		echo -e "\nThe following DNS servers have troubles on SOA queries: \n$hang\n"
	else
		echo $hang | mailx -s "SOA query hang" $MAIL
	fi
fi
